import re,os,chardet

def cellSentence(sentence):
    pat=r'[“「]([^」”]+)[”」]'
    sentences=re.findall(pat,sentence)
    return ''.join(sentences)

def abstract(path):
    allowTypes=['txt','xhtml','html']
    if path.split('.')[-1] not in allowTypes:
        return []

    try:
        with open(path,'r',encoding='utf8') as f:
            text=f.readlines()
    except:
        try:
            with open(path,'r',encoding='gbk') as f:
                text=f.readlines()
        except:
            print('warn: '+path)
            return []
    # 过滤html标签
    html_tags_pat=r'<[/]?[^>/]+[/]?>'
    text=[re.sub(html_tags_pat,'',_cell) for _cell in text if text!='']

    collections=[]
    for _cell in text:
        sentence=cellSentence(_cell)
        if len(sentence.replace(' ',''))>0:
            collections.append(sentence)
    return collections

def getEncoding(filePath):
    with open(filePath,'rb') as f:
        tmp=chardet.detect(f.read())['encoding']
        print(tmp)
        return tmp

root='./bookself/'
fs=os.walk(root).__next__()
subs=fs[1]
raws=[]
for sub in subs:
    if sub=='stop':
        continue
    path=root+sub+'/'
    _tmp_fs=os.walk(path)
    for _f in _tmp_fs:
        fileRoot=_f[0]
        for fileName in _f[-1]:
            filePath=fileRoot+'/'+fileName
            raws+=['-']+abstract(filePath)


def apply_filter(li):
    i=0
    while i<len(li)-1:
        if li[i]==li[i+1]:
            li.pop(i)
            i-=1
        else:
            i+=1

apply_filter(raws)

print(len(raws))
with open('./raw/o2.txt','w+',encoding='utf8') as f:
    f.write('\n'.join(raws))